# coding:utf8

import UrlManage,DownloadManage,ParseManage,DataCollectManage

class SpiderBase(object):

	def __init__(self):
		# URL管理器
		self.urlManage = UrlManage.UrlManage()
		# 页面下载器
		self.downloadManage = DownloadManage.DownloadManage()
		# 页面解析器
		self.parseManage = ParseManage.ParseManage()
		# 数据处理器
		self.dataCollectManage = DataCollectManage.DataCollectManage()

	def work(self,url,num=1000):	
		# 第一步将要处理的URL交给URL管理器 格式化URL并且管理URL
		self.urlManage.addUrl(url)
		count = 1
		while self.urlManage.hasUrl():
			try:
				# 第二步从URL管理器中获取一条可以下载的URL
				newUrl = self.urlManage.getOneUrl()
				print("%d %s" %(count,newUrl))
				# 第三步下载数据
				content = self.downloadManage.do(newUrl)
				if content is None:
					continue
				# 第四步将下载的数据交给页面解析器，返回页面数据和页面中的URLS
				data,urls = self.parseManage.do(newUrl,content)
				# 第五部将解析出的数据交给数据处理器，将URLS交给URL管理器
				if data is None:
					continue
				self.dataCollectManage.do(data)
				self.urlManage.addUrls(urls)
				if count == num:
					break
				count = count + 1
			except AttributeError as ae:
				print("error_message:%s" %(ae.getMessage()))
			except Exception as e:
				print("error_message:%s" %(e.getMessage()))
		self.urlManage.allow = set()
		self.urlManage.disable = set()
				
			


if __name__ == "__main__":
	url = 'https://www.tepintehui.com/detail/112535?ce'
	#url = 'https://www.tepintehui.com/detail/112535%23comment-list-ele'
	spider = SpiderBase()
	spider.work(url,10)